#!/usr/bin/python

"""A scraper for formspring.me that saves all of a users questions and
answers. It stores them into a sqlite database. To see how to use this
script, run it without any arguments.

When the formspring API goes public, remove the 'beta-' from the variable
called API_LOCATION below.
"""
__author__ = 'meltingwax@gmail.com'

API_LOCATION = 'http://beta-api.formspring.me'


import os
import sys
import urllib2


try:
    import json
except ImportError:
    try:
        import simplejson as json
    except ImportError:
        print 'Could not find json module. Try installing simplejson: '
        print '  http://code.google.com/p/simplejson/'
        sys.exit(1)

import sqlite3


def main():
    username, db_fname = parse_args()

    if os.path.exists(db_fname):
        print 'sqlite database', db_fname, 'exists, would you like to overwrite (Y/n)?',
        answer = None

        while True:  # a valid answer either breaks the loop or exits the program
            if answer is not None:
                print 'Invalid response. Enter (Y/n):',
            answer = raw_input()
            if answer == 'Y':
                os.remove(db_fname)
                break
            elif answer == 'n':
                print 'Bye.'
                sys.exit(1)

    db_conn = sqlite3.connect(db_fname)
    db = db_conn.cursor()

    db.execute("""
        CREATE TABLE questions (
            id INTEGER PRIMARY KEY,
            question TEXT,
            answer TEXT,
            time TEXT,
            username TEXT
        )""")
    db_conn.commit()
    
    last_min_id = None

    while True:  # breaks when recieves empty result list, or error
        list_url = '%s/answered/list/%s.json' % (API_LOCATION, username)
        if last_min_id is not None:
            list_url += '?max_id=%d' % last_min_id
    
        api_resp = urllib2.urlopen(list_url).read()
        resp_dict = json.loads(api_resp)

        if resp_dict['status'] == 'error':
            print 'The API returned an error, halting: %s' % resp_dict['error']
            sys.exit(1)

        items_list = resp_dict['response']
        if items_list:
            print 'Fetched %d items' % len(items_list)
        else:
            break
        
        for item in items_list:
            insert_item(item, db)

        db_conn.commit()
            
        last_min_id = int(items_list[-1]['id'])

    print 'All done.'
        

def insert_item(item, db):
    item_id = int(item['id'])
    question = item['question']
    answer = item['answer']
    time = item['time']

    if isinstance(item['asked_by'], dict):
        username = item['asked_by']['username']
    else:
        username = ''

    db.execute('INSERT INTO questions (id, question, answer, time, username) '
               'VALUES (?, ?, ?, ?, ?)',
               (item_id, question, answer, time, username))
                

def parse_args():
    args = sys.argv[1:]

    if len(args) != 2:
        print '%s <username> <sqlite_db>' % sys.argv[0]
        sys.exit(1)
    else:
        return args


if __name__ == '__main__':
    main()
